# -*- coding=utf-8 -*-

'''
下载文章
先拿头条开练
后续csdn、zhihu
'''
import re,requests,time,json
import pprint as pp
PATH_ROOT = 'C:/E/temp/pic/'
VIDEO_SAVE_PATH = 'c:/e/temp/video/'

HEADER = {
    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
    #,"sec-ch-ua": 'Google Chrome";v="87", " Not;A Brand";v="99", "Chromium";v="87'
    ,"Referer": "https://www.youtube.com/"
}

#获取要下载文章网页的url列表list
def getVideoUrlList():
    urlList = []
    with open('content.txt','r') as urlFile:
        lines = urlFile.readlines()
        for _l in lines:
            if re.match(r'^http[s]{0,1}://',_l):
                urlList.append(_l.replace('\n','').replace('\r',''))
    return urlList

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
#from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
def getHtmlOfWebPage(url:str):
    #------------------------- selenium begin -------------------------#
    # 使用headless无界面浏览器模式
    chrome_options = Options()
    chrome_options.add_argument('--headless') #增加无界面选项
    chrome_options.add_argument('--disable-gpu') #如果不加这个选项，有时定位会出现问题
    chrome_options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"')
    #chrome_options.add_argument('Referer:https://www.youtube.com/')
    chrome_options.add_argument('Referer:https://www.toutiao.com/')
    chrome_options.add_argument('x-request-with:XMLHttpRequest')
    chrome_options.add_argument(':method:GET')
    chrome_options.add_argument(':scheme:https')
    chrome_options.add_argument('sec-fetch-dest:document')
    chrome_options.add_argument('sec-fetch-mode:no-cors')
    chrome_options.add_argument('sec-fetch-site:none')
    chrome_options.add_argument('sec-fetch-user:?1')
    chrome_options.add_argument('sec-fetch-site:cross-site')
    chrome_options.add_argument('upgrade-insecure-request:1')
    #chrome_options.add_argument('cookie:')
    chrome_options.add_experimental_option('excludeSwitches',['enable-logging','enable-automation'])
    # 启动浏览器，获取网页源代码 chromedriver.exe放在phantomjs的bin目录下一起加入path变量
    serviceDriver = Service(executable_path='chromedriver.exe')
    browser = webdriver.Chrome(service=serviceDriver,options=chrome_options) 
    browser.get(url)
    #time.sleep(5)
    browser.set_page_load_timeout(5)
    html = browser.page_source
    browser.quit()
    #------------------------- selenium end -------------------------#
    return html

#解析页面得到音频与视频url字典集合
#当前可得到 youtube.com 网站的视频
#update time：2022-04-18 (开发中……)
def getUrlsDictByHtmlForYoutubeCom(html:str):
    video_info_dict = {}
    #bsp = BeautifulSoup(html,'html.parser')
    #获取视频封面
    thumbnailUrl = re.findall('<link itemprop="thumbnailUrl" href="(.+?)">',html)
    video_info_dict['thumbnailUrl'] = thumbnailUrl[0] if len(thumbnailUrl)>0 else '' 
    #获取视频信息
    json_content = re.findall('var ytInitialPlayerResponse = (.+?)}}}};',html) 
    if len(json_content) > 0 :
        json_content = json_content[0] + '}}}}'
        target_json_content = json.loads(json_content)
        video_info_dict['thumbnailUrlList'] = target_json_content['videoDetails']['thumbnail']['thumbnails'] #这里获取的是封面图列表 包含不同尺寸由小到大排列的列表list
        if '' == video_info_dict['thumbnailUrl'] and len(video_info_dict['thumbnailUrlList'])>0:
            video_info_dict['thumbnailUrl'] = video_info_dict['thumbnailUrlList'][-1]['url'] #获取最大图的url list中最后一个字典元素的key='url'的值
        video_info_dict['title'] = delSpecialMarkOfTitle(target_json_content['videoDetails']['title']) #获取视频标题

    return video_info_dict

#下载资源
def downloadByUrl(url,file_name):
    if not re.match('^http[s]{0,1}://',url):
        #return
        url = r'https:'+url
    if requests.head(url).status_code != 200:
        return
    _response = requests.get(url,headers=HEADER)
    _content = _response.content
    _file = file_name
    with open(_file,'wb') as _f :
        _f.write(_content)

#去掉不能在文件名中出现的和不想要的字符
def delSpecialMarkOfTitle(title:str):
    return re.sub(r'[\|:/<>?"* #\\]','',title)

#目前对Youtube网站只获取视频标题和封面url 不下载视频和封面 update：2022-04-21
def getVideoFromWebPageUrl(url:str):
    html = getHtmlOfWebPage(url)
    video_dict = getUrlsDictByHtmlForYoutubeCom(html)
    #with open('youtube-thumbnail.json','a+',encoding='utf-8') as _f :
    #    _f.write(json.dumps(video_dict,indent=2)+'\n')
    #openWebPage(video_dict['thumbnailUrl'])

#------------------------- selenium begin -------------------------#
#browser设为全局变量可保持浏览器窗口一直处在开启状态 可打开多页签
# 使用headless无界面浏览器模式
#my_chrome_options = Options()
##my_chrome_options.add_argument('--headless') #增加无界面选项
#my_chrome_options.add_argument('--disable-gpu') #如果不加这个选项，有时定位会出现问题
#my_chrome_options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"')
#my_chrome_options.add_argument('Referer:https://www.toutiao.com/article/7091553329775526412/')
#my_chrome_options.add_argument('x-request-with:XMLHttpRequest')
#my_chrome_options.add_argument('cookie: msToken=-ZnF-uF0TIHbObwGesKXgnqFxREhDwdhBOU6Nm5t3_WIlwwR6zjb56Wkrr3nA4oiTEyoqxc6AVOinT7fAHYVF2iJnt3u69BIJ8v060xaa5ko; __ac_signature=_02B4Z6wo00f01itNYDAAAIDCq0-ac7xX-LorbWSAAOjD09; tt_webid=7090359687744620040; ttcid=e2cfc1ca0ade4f9390439bcf0a914c4b13; csrftoken=e9801ed59700aa0bd9bfe098881e22a6; _tea_utm_cache_24=undefined; passport_csrf_token=ee69c453c6dd43f03529c3af1537fa40; passport_csrf_token_default=ee69c453c6dd43f03529c3af1537fa40; s_v_web_id=verify_92a93d911a3dba37de07367df4a643e5; _tea_utm_cache_2018=undefined; n_mh=A75IbYmG5Als4t19av_R1wHaT665qkHJvxIEhqZGcAY; passport_auth_status=95f6dd7b5fca5a78891fe573c4089d34%2C; passport_auth_status_ss=95f6dd7b5fca5a78891fe573c4089d34%2C; sso_auth_status=18d3c10c48231f5c07ab62e6146b1fbd; sso_auth_status_ss=18d3c10c48231f5c07ab62e6146b1fbd; sso_uid_tt=1d8ba690d0c9402b119d7f6d75c6b3a0; sso_uid_tt_ss=1d8ba690d0c9402b119d7f6d75c6b3a0; toutiao_sso_user=d4a0cb1048b74827fd39314c56fb6a99; toutiao_sso_user_ss=d4a0cb1048b74827fd39314c56fb6a99; sid_ucp_sso_v1=1.0.0-KDg3MTFjZjgzZjM1NjZkNmMwNGRlNzFkYTdjMWQyYzU5YzMyM2NlMDkKFAjO5fTlmwMQwImYkwYYGDgCQPEHGgJsZiIgZDRhMGNiMTA0OGI3NDgyN2ZkMzkzMTRjNTZmYjZhOTk; ssid_ucp_sso_v1=1.0.0-KDg3MTFjZjgzZjM1NjZkNmMwNGRlNzFkYTdjMWQyYzU5YzMyM2NlMDkKFAjO5fTlmwMQwImYkwYYGDgCQPEHGgJsZiIgZDRhMGNiMTA0OGI3NDgyN2ZkMzkzMTRjNTZmYjZhOTk; odin_tt=4299460d517d201d3d8fd0ab3d145dd014ae917e8807026b43f9308d3b702520fc2658056defc7a906f3094c5a28b240; sid_guard=d4a0cb1048b74827fd39314c56fb6a99%7C1650853057%7C5184000%7CFri%2C+24-Jun-2022+02%3A17%3A37+GMT; uid_tt=1d8ba690d0c9402b119d7f6d75c6b3a0; uid_tt_ss=1d8ba690d0c9402b119d7f6d75c6b3a0; sid_tt=d4a0cb1048b74827fd39314c56fb6a99; sessionid=d4a0cb1048b74827fd39314c56fb6a99; sessionid_ss=d4a0cb1048b74827fd39314c56fb6a99; sid_ucp_v1=1.0.0-KDU2NTEwNjEzYTAxMjUxODdjMDFmY2ExNzExZGQ5ZTI5MGYyMjQ4MTYKFgjO5fTlmwMQwYmYkwYYGCAMOAJA8QcaAmxxIiBkNGEwY2IxMDQ4Yjc0ODI3ZmQzOTMxNGM1NmZiNmE5OQ; ssid_ucp_v1=1.0.0-KDU2NTEwNjEzYTAxMjUxODdjMDFmY2ExNzExZGQ5ZTI5MGYyMjQ4MTYKFgjO5fTlmwMQwYmYkwYYGCAMOAJA8QcaAmxxIiBkNGEwY2IxMDQ4Yjc0ODI3ZmQzOTMxNGM1NmZiNmE5OQ; local_city_cache=%E4%BF%9D%E5%AE%9A; tt_anti_token=MoScIxvAxzGM9gyT-c9c23ed24918eecad9af9d0607a1cd8ef64429e6a055c3dee3dc66d897d7edc6; tt_scid=k-jOvUuGWXrKiLz.q5JnO0I3pkVp.FOM18TBFptyZI8f9E0T7Kz4PVjlQN1vQCOA2f34; ttwid=1%7C5LahyDJuvNh24YqfXZd8v2MiDZ9nSLv5CYc59212LJg%7C1651197825%7Cec2e1f67d15d69621d189a1b0457869e7139424e63a77c2497a60e6e063862a9; MONITOR_WEB_ID=fab3969c-cd5e-4305-9b83-73b37eaae70a')
#my_chrome_options.add_experimental_option('excludeSwitches',['enable-logging','enable-automation'])
## 启动浏览器，获取网页源代码 chromedriver.exe放在phantomjs的bin目录下一起加入path变量
#my_serviceDriver = Service(executable_path='chromedriver.exe')
#my_browser = webdriver.Chrome(service=my_serviceDriver,options=my_chrome_options)
#my_browser.get('https://www.sogou.com/')
#my_browser_js = 'window.open("{}","_blank");'
#------------------------- selenium end -------------------------#
#def openWebPage(url:str):
#    my_browser.execute_script('window.open("{}","_blank");'.format(url))
#    ActionChains(my_browser).key_down(Keys.CONTROL).send_keys('t').key_up(Keys.CONTROL).perform()

def mainDownload():
    uList = getVideoUrlList()
    for _url in uList:
        if not re.match('^http[s]{0,1}://',_url):
            continue
        getVideoFromWebPageUrl(_url)

import pprint,os,imghdr,time
from shutil import copyfile
from PIL import Image
#判断图片类型
def whatTypeImg(_img):
    return imghdr.what(_img)
#判断图片类型是否与扩展名匹配 不匹配则修改扩展名
def replaceSuffix(_img):
    _img_temp = _img.split('.')
    if len(_img_temp) > 1 :
        _target_img = _img
        sourceImg = _img
        targImg = _target_img
        _source_suffix = _img_temp[1]
        _current_img_type = whatTypeImg(sourceImg)
        if _source_suffix != _current_img_type and _current_img_type != None:
            targImg = targImg.replace('.'+_source_suffix,'.'+_current_img_type)
            copyfile(sourceImg,targImg)
            os.remove(sourceImg)          
        #if whatTypeImg(targImg) == 'gif':
        #    img = Image.open(targImg).convert('RGB')
        #    img.save(targImg.replace('.gif','.jpeg'),'JPEG')
        #    os.remove(targImg)
#给序号拼’0’ 方便排序
BIT_SUM = 2 #序号位数
def penddingIndex(_index):
    strIndex = str(_index)
    strIndexLen = len(strIndex)
    if len(strIndex) < BIT_SUM :
        return '0'*(BIT_SUM - strIndexLen) + strIndex
    return strIndex
#由正则表达式从html文本中选取图片url 然后下载图片
def getPicVieRegxFromHtml(rePattern,html):
    #当前日期、时刻
    file_name = time.strftime('%Y-%m-%d-%H%M%S',time.localtime())
    urls = re.findall(rePattern,html)
    index = 0
    for _url in urls:
        _u = re.sub('^\/\/','https://',_url[0])
        print(_u,_url[1])
        downloadByUrl(_u,PATH_ROOT+file_name+penddingIndex(index)+'.jpg')
        index += 1
    for pic in os.listdir(PATH_ROOT):
        replaceSuffix(PATH_ROOT+pic)
if __name__ == '__main__':
    print('----------------begin----------------')
    #mainDownload()
    #--------------调试代码 begin------------#
    #html = getHtmlOfWebPage(r'https://www.toutiao.com/article/7091553329775526412/')
    #html = openWebPage(r'https://www.toutiao.com/article/7091553329775526412/')
    #with open('toutiao.html','w',encoding='utf-8') as _f :
    #    _f.write(html)
    with open('toutiao.html','r',encoding='utf-8') as _f :
        html = _f.read()
        #getPicVieRegxFromHtml('<div class="pgc-img"><img src="(.+?)" img_width="(\d+?)" img_height="(\d+?)" alt="(.+?)" inline="(\d+?)">',html)
        getPicVieRegxFromHtml('<img class="weitoutiao-img" src="(.+?)"(/{1}?)>',html)
        getPicVieRegxFromHtml('<img src="(.+?)"( {1}?)',html)
    #getUrlsDictByHtmlForYoutubeCom(html)
    #--------------调试代码 end  ------------#
    print('---------------- end ----------------')

'''
        #---------临时调试 后期重构 begin-----------#
        suffix = re.findall('\.jpg|\.jpeg|\.png|\.webp',video_info_dict['thumbnailUrl'])
        suffix = suffix[0] if len(suffix)>0 else '.jpeg'
        print(video_info_dict['title'],video_info_dict['thumbnailUrl'])
        downloadByUrl(video_info_dict['thumbnailUrl'],PATH_ROOT+video_info_dict['title']+suffix)
        #---------临时调试 后期重构 end  -----------#
'''
